﻿using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using CMN.BLL;
using CMN.Common;
using CMN.Entity;
using System.Text.RegularExpressions;
using System.Threading;
using System.Web;
using System.Net;

using System.Data;
using System.Drawing; 
using System.Drawing.Imaging;
namespace CMN.Extends
{
    public  class CollectHelper
    {


        protected   Regex regValue;
        protected   Regex reg;

        #region 获取采集连接
        public List<Extend_CollectLinkInfoEntity> GetLinks(Extend_CollectSetInfoEntity setLinksInfo)
        {
           
            
           // Uri baseUri = new Uri(setLinksInfo.Url);
           
            List<Extend_CollectLinkInfoEntity> collectLinkList=new List<Extend_CollectLinkInfoEntity>();
           
                if(setLinksInfo.StartNum>setLinksInfo.EndNum)
                    for (int i = setLinksInfo.StartNum; i >=setLinksInfo.EndNum; i = i + setLinksInfo.AddNum)
                    {
                        GetLink(collectLinkList, i, setLinksInfo);
                        System.Threading.Thread.Sleep(20);
                    }
                else
                    for (int i = setLinksInfo.StartNum; i <= setLinksInfo.EndNum; i = i + setLinksInfo.AddNum)
                    {
                        GetLink(collectLinkList, i, setLinksInfo);
                        System.Threading.Thread.Sleep(20);
                    }
                    
           
            return collectLinkList;
        }
        public void GetLink(List<Extend_CollectLinkInfoEntity> collectLinkList, int i, Extend_CollectSetInfoEntity setLinksInfo)
        {
            string strUrl = string.Empty;

            // Uri baseUri = new Uri(setLinksInfo.Url);
            string content="";

            try
            {


                if (i == setLinksInfo.StartNum && !string.IsNullOrEmpty(setLinksInfo.FirstUrl))
                {
                    strUrl = setLinksInfo.FirstUrl;
                }
                else
                    strUrl = setLinksInfo.Url.Replace("{*}", i.ToString());
                if (!string.IsNullOrEmpty(strUrl.Trim()))
                {
                    try
                    {
                        try
                        {
                            content = Utils.GetRequestString(strUrl, 16000, 1, Encoding.GetEncoding(setLinksInfo.Encoding), (string.IsNullOrEmpty(setLinksInfo.Cookie) ? null : setLinksInfo.Cookie));

                        }
                        catch (Exception ex)
                        {
                            AppLog.Write(ex.Message+strUrl, AppLog.LogMessageType.Error, ex);
                        }
                        if (string.IsNullOrEmpty(content.Trim()))
                        {
                            AppLog.Write("GetLink-content:内容为空", AppLog.LogMessageType.Debug);
                            return;
                        }
                        string strBaseHref = GetSingleTagValue(content, "base", "href");
                        if (!string.IsNullOrEmpty(strBaseHref))
                        {
                            strUrl = strBaseHref;
                            // baseUri = new Uri(strBaseHref);
                        }
                        if (!string.IsNullOrEmpty(setLinksInfo.ListMatchLeft))
                            content = GetCutString(content, setLinksInfo.ListMatchLeft, setLinksInfo.ListMatchRight, true);
                        List<Extend_CollectLinkInfoEntity> collectLinkList2 = new List<Extend_CollectLinkInfoEntity>();
                        //if (string.IsNullOrEmpty(content))
                        //{
                        //    AppLog.Write("GetLink-GetCutString-content内容为空", AppLog.LogMessageType.Debug);
                        //}
                        if (string.IsNullOrEmpty(setLinksInfo.UrlRegex))
                        {
                            collectLinkList2 = GetLinkHref(content, setLinksInfo.IncludeStr, setLinksInfo.WithoutStr, strUrl);
                        }
                        else
                            collectLinkList2 = GetLinkRegexHref(content, setLinksInfo.IncludeStr, setLinksInfo.WithoutStr, setLinksInfo.UrlRegex, strUrl);
                        bool isHas = false;
                        foreach (Extend_CollectLinkInfoEntity linkInfo2 in collectLinkList2)
                        {
                            if (collectLinkList.Contains(linkInfo2))
                                isHas = true;
                            //foreach (Extend_CollectLinkInfoEntity linkInfo in collectLinkList)
                            //{
                            //    if (linkInfo2.Url == linkInfo.Url)
                            //    {
                                   
                            //        isHas = true;
                            //        break;
                            //    }
                            //}

                            if (!isHas)
                            {
                                collectLinkList.Add(linkInfo2);

                            }
                            isHas = false;
                        }
                    }
                    catch (Exception ex)
                    {
                        AppLog.Write(ex.Message, AppLog.LogMessageType.Error, ex);
                    }
                }

            }

            catch (Exception ex)
            {
                AppLog.Write("采集列表错误" + strUrl, AppLog.LogMessageType.Error, ex);
            }
        }
        #endregion

        /// <summary>
        /// 内容分页组合
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public Extend_CollectContentEntity GetPagingContent(Extend_CollectSetInfoEntity setLinksInfo, string url)
        {
            Extend_CollectContentEntity collectContent = new Extend_CollectContentEntity();
            List<Extend_CollectLinkInfoEntity> linkList = new List<Extend_CollectLinkInfoEntity>();
            string reContent = string.Empty;
            string reTitle = string.Empty;
            if (string.IsNullOrEmpty(setLinksInfo.PagingReg))
            {
                throw new Exception("内容分页正则表达式为空");
            }
            Encoding coding = null;
            if (string.IsNullOrEmpty(setLinksInfo.ContentEncoding))
            {
                coding = Encoding.GetEncoding(setLinksInfo.Encoding);
            }
            else
                coding = Encoding.GetEncoding(setLinksInfo.ContentEncoding);

            string content = string.Empty;
            try
            {
                content = Utils.GetRequestString(url, 16000, 1, coding, (string.IsNullOrEmpty(setLinksInfo.Cookie) ? null : setLinksInfo.Cookie), setLinksInfo.FirstUrl);
            }
            catch (Exception ex)
            {
                AppLog.Write(ex.Message + url, AppLog.LogMessageType.Error);
            }
            if (setLinksInfo.PagingReg.StartsWith("jsregex"))
            {
                linkList = GetLinkRegexHref(content, setLinksInfo.PagingReg, url);
            }
            else
                linkList = GetLinkHref(content, setLinksInfo.PagingReg, "", url.Substring(0, (url.LastIndexOf('/') + 1)));

            collectContent = GetContent(setLinksInfo, content, true);
            reContent = collectContent.TextContent;
            bool isHas = false;
            for (int i = 0; i < linkList.Count; i++)
            {


                if (linkList[i].IsCollect == false && linkList[i].Url != "")
                {
                    string contentTemp = Utils.GetRequestString(linkList[i].Url, 16000, 1, coding, (string.IsNullOrEmpty(setLinksInfo.Cookie) ? null : setLinksInfo.Cookie));
                    List<Extend_CollectLinkInfoEntity> linkListTemp = new List<Extend_CollectLinkInfoEntity>();
                    linkListTemp = GetLinkHref(contentTemp, setLinksInfo.PagingReg, "", linkList[i].Url.Substring(0, (linkList[i].Url.LastIndexOf('/') + 1)));

                    foreach (Extend_CollectLinkInfoEntity linkInfo in linkListTemp)
                    {
                        foreach (Extend_CollectLinkInfoEntity linkInfo2 in linkList)
                        {
                            if (linkInfo.Url == linkInfo2.Url)
                            {
                                isHas = true;
                                break;
                            }
                        }
                        if (!isHas)
                        {
                            linkList.Add(linkInfo);

                        }
                        isHas = false;
                    }

                    reContent = string.Concat(reContent, GetContent(setLinksInfo, contentTemp, true).TextContent);
                    linkList[i].IsCollect = true;
                }
            }

            collectContent.TextContent = reContent;

            return collectContent;
        }

        #region  匹配所有绝对超链接地址
       /// <summary>
        /// 获取js生成分页连接
       /// </summary>
       /// <param name="inputstring"></param>
        /// <param name="urlRegex">分隔符jsregex</param>
       /// <param name="baseUrl"></param>
       /// <returns></returns>
        public List<Extend_CollectLinkInfoEntity> GetLinkRegexHref(string inputstring,  string urlRegex, string baseUrl)
        {
            
            string a_url = string.Empty;
            List<Extend_CollectLinkInfoEntity> collectLinkList = new List<Extend_CollectLinkInfoEntity>();
            bool isHas = false;
            Regex regex = null;
            //申明和实例化一个正则表达式对象，这里要加上参数RegexOptions.IgnoreCase，用于忽略文本大小写
            string[] strRegex = urlRegex.Split( new string[]{"jsregex"}, StringSplitOptions.None);
            if (strRegex.Length != 5)
            {
                return collectLinkList;
            }

            int pageCount = 0;
            string urlPageCountRegex = strRegex[1];

            //去除不变地址后面的
            string urlRepalceRegex = strRegex[2];
            //分页地址分隔符
            string pageSplit = strRegex[3];
            //后缀，如.html
            string urlSuffix = strRegex[4];
            
            regex = new Regex(urlPageCountRegex, RegexOptions.IgnoreCase);

            pageCount = Utils.StringToInt(regex.Match(inputstring).Value, 1);
            //上处可以匹配文本里有多组相似的结果全部结果，如果只有一组的话，用Match match = regex.Match(json);
            if (pageCount > 1)
            {
                string strRoundUrl = "";
                regex = new Regex(urlRepalceRegex, RegexOptions.IgnoreCase);
                strRoundUrl = regex.Replace(baseUrl, "");
                for (int i = 2; i <= pageCount;i++ )
                {

                    a_url = string.Concat(strRoundUrl, pageSplit, i.ToString(), urlSuffix);
                   

                    Extend_CollectLinkInfoEntity lInfo = new Extend_CollectLinkInfoEntity();

                    a_url = Utils.GetAbsolutePath(baseUrl, a_url);
                    lInfo.Url = a_url;
                    lInfo.CollectNum = 0;
                    lInfo.IsCollect = false;
                    foreach (Extend_CollectLinkInfoEntity linkInfo in collectLinkList)
                    {
                        if (lInfo.Url == linkInfo.Url)
                        {
                            isHas = true;
                            break;
                        }
                    }
                    if (!isHas)
                    {
                        collectLinkList.Add(lInfo);

                    }
                    isHas = false;


                }
            }

            return collectLinkList;

        }
        /// <summary>
        /// 匹配所有绝对超链接地址
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="str_has">超链接中包括</param>
        /// <param name="oldUrl">获取地址</param>
        /// <returns></returns>
        public List<Extend_CollectLinkInfoEntity> GetLinkRegexHref(string inputstring, string str_has, string strNoHas, string urlRegex, string baseUrl)
        {
          
            string a_url = string.Empty;
            List<Extend_CollectLinkInfoEntity> collectLinkList = new List<Extend_CollectLinkInfoEntity>();
            bool isHas = false;



            Regex regex = new Regex(urlRegex, RegexOptions.IgnoreCase);
            //申明和实例化一个正则表达式对象，这里要加上参数RegexOptions.IgnoreCase，用于忽略文本大小写

            MatchCollection matches = regex.Matches(inputstring);
            //上处可以匹配文本里有多组相似的结果全部结果，如果只有一组的话，用Match match = regex.Match(json);
           
            foreach (Match match in matches)
            {
                a_url = match.Groups["url"].Value.Replace("\\", "");//此处Name是匹配表达式(?'Name'[^"]*)里的Name，显式分组


                regValue = new Regex("#[^# ]*[ ]*", RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_url))
                {
                    a_url = a_url.Replace(regValue.Match(a_url).Value, "");
                }
                regValue = new Regex(str_has, RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_url))
                {
                    regValue = new Regex(strNoHas, RegexOptions.IgnoreCase);
                    if (String.IsNullOrEmpty(strNoHas) || !regValue.IsMatch(a_url))
                    {
                        Extend_CollectLinkInfoEntity lInfo = new Extend_CollectLinkInfoEntity();

                        a_url = Utils.GetAbsolutePath(baseUrl, a_url);
                        lInfo.Url = a_url;
                        lInfo.CollectNum = 0;
                        lInfo.IsCollect = false;
                        foreach (Extend_CollectLinkInfoEntity linkInfo in collectLinkList)
                        {
                            if (lInfo.Url == linkInfo.Url)
                            {
                                isHas = true;
                                break;
                            }
                        }
                        if (!isHas)
                        {
                            collectLinkList.Add(lInfo);

                        }
                        isHas = false;
                    }
                }

            }

            return collectLinkList;

        }
      
        /// <summary>
        /// 匹配所有绝对超链接地址
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="str_has">超链接中包括</param>
        /// <param name="oldUrl">获取地址</param>
        /// <returns></returns>
        public List<Extend_CollectLinkInfoEntity> GetLinkHref(string inputstring, string str_has,string strNoHas, string baseUrl)
        {
            Common.RegExpression.LinkRegExpression linkRg = new CMN.Common.RegExpression.LinkRegExpression();
            Common.RegExpression.RegContext regContent = new CMN.Common.RegExpression.RegContext(inputstring, 'M', "");
            linkRg.Evaluate(regContent);
            Common.RegExpression.IRegExpression reg;
            Common.RegExpression.RegContext regTemp;
            string a_url = string.Empty;
            List<Extend_CollectLinkInfoEntity> collectLinkList = new List<Extend_CollectLinkInfoEntity>();
            bool isHas = false;
            string[] strUrlList=null;
            foreach (string strmatch in regContent.Matches)
            {
                reg = new CMN.Common.RegExpression.HrefRegExpression();
                regTemp = new CMN.Common.RegExpression.RegContext(strmatch, 'M', "");
             

                if (reg.IsMatch(strmatch))
                {
                    reg.Evaluate(regTemp);
                    a_url = regTemp.ToString();
                    strUrlList=a_url.Split( new string[]{"href="},StringSplitOptions.RemoveEmptyEntries);
                    if (strUrlList.Length > 1)
                    {
                        a_url = strUrlList[0];
                    }
                    a_url = a_url.Replace("href=", "").Replace("HREF=", "").Replace("'", "").Replace("\"", "");
                }
               //去除超链接#后面内容
                //regValue = new Regex("#[^# ]*[ ]*", RegexOptions.IgnoreCase);
                //if (regValue.IsMatch(a_url))
                //{
                //    a_url = a_url.Replace(regValue.Match(a_url).Value, "");
                //}
                regValue = new Regex(str_has, RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_url) )
                {
                    regValue = new Regex(strNoHas, RegexOptions.IgnoreCase);
                    if (String.IsNullOrEmpty(strNoHas) || !regValue.IsMatch(a_url))
                    {
                        Extend_CollectLinkInfoEntity lInfo = new Extend_CollectLinkInfoEntity();
                        
                        a_url = Utils.GetAbsolutePath(baseUrl, a_url);
                        lInfo.Url = a_url;
                        lInfo.CollectNum = 0;
                        lInfo.IsCollect = false;
                        if (!collectLinkList.Contains(lInfo))
                        {
                            collectLinkList.Add(lInfo);

                        }
                        isHas = false;
                    }
                }
               
            }
            return collectLinkList;

        }
        #endregion

        #region 获取类似第一个meta的标签
        /// <summary>
        /// 获取类似第一个meta的标签
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="tagName">获取标签名</param>
        /// <param name="attrname">标签类型如:name</param>
        /// <param name="key">标签类型值如:keywords</param>
        /// <returns>字符串</returns>
        protected string GetSingleTagValue(string inputstring, string tagName, string attrname, string key)
        {
            Regex reg = new Regex("<[ ]*" + tagName + " [^<>]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string result = string.Empty;
            foreach (Match match in matchs)
            {
                string matchValue = match.Value;
                regValue = new Regex("content=[\"|'][^'\"]*[\"|']", RegexOptions.IgnoreCase);
                if ((matchValue.ToLower().IndexOf(attrname.ToLower() + "=\"" + key.ToLower() + "\"") != -1) || (matchValue.ToLower().IndexOf(attrname.ToLower() + "='" + key.ToLower() + "'") != -1))
                {
                    if (regValue.IsMatch(matchValue))
                    {
                        result = regValue.Match(matchValue).Value;
                        if (!string.IsNullOrEmpty(result))
                        {
                            result = result.Replace("CONTENT=", "").Replace("content=", "").Replace("\"", "").Replace("'", "").Trim();
                        }
                    }
                    return result;
                }
            }
            return string.Empty;
        }


        /// <summary>
        /// 获取类似第一个meta的标签
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="tagName">获取标签名</param>
        /// <param name="attrname">标签类型如:name</param>
        /// <param name="key">标签类型值如:keywords</param>
        /// <returns>字符串</returns>
        protected string GetSingleTagValue(string inputstring, string tagName, string attrname)
        {
            Regex reg = new Regex("<[ ]*" + tagName + " [^<>]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string result = string.Empty;
            foreach (Match match in matchs)
            {
                string matchValue = match.Value;
                regValue = new Regex(attrname + "=[\"|'][^'\"]*[\"|']", RegexOptions.IgnoreCase);

                if (regValue.IsMatch(matchValue))
                {
                    result = regValue.Match(matchValue).Value;
                    if (!string.IsNullOrEmpty(result))
                    {
                        result = result.Replace(attrname + "=", "").Replace("\"", "").Replace("'", "").Trim();
                    }
                }
                return result;

            }
            return string.Empty;
        }
        #endregion
        /// <summary>
        /// 获取类似title标签的值
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="tagName">获取标签名</param>
        /// <returns>返回字符串</returns>
        protected string GetFixTagContent(string inputstring, string tagName)
        {
            reg = new Regex("<[ ]*" + tagName + ".+?" + tagName + "[ ]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string result = string.Empty;
            string a_content = string.Empty;
            string a_content_left = string.Empty;
            string a_content_right = string.Empty;
            foreach (Match match in matchs)
            {
                a_content = match.Value;

                regValue = new Regex("<[ ]*" + tagName + "[^<>]*>", RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_content))
                {
                    a_content_left = regValue.Match(a_content).Value;
                }
                regValue = new Regex("</[ ]*" + tagName + "[ ]*>", RegexOptions.IgnoreCase);
                if (regValue.IsMatch(a_content))
                {
                    a_content_right = regValue.Match(a_content).Value;
                }
                if (a_content.Length > 0 && a_content_left != "" && a_content_right != "")
                {
                    result = a_content.Replace(a_content_left, "").Replace(a_content_right, "");
                }
                return result;
            }
            return result;
        }
        /// <summary>
        /// 返回超链接文本
        /// </summary>
        /// <param name="inputstring">含有连接的字符窜</param>
        /// <param name="strInclude">超链接中包含</param>
        /// <returns>string</returns>
        protected List<string> GetLinkText(string inputstring, string strInclude,Int32 maxListLength)
        {
            reg = new Regex("<[ ]*a.+?a[ ]*>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            string a_content = string.Empty;
          
            List<string> lst = new List<string>();
            foreach (Match match in matchs)
            {
                a_content = match.Value;

                if (a_content.ToLower().IndexOf(strInclude.ToLower()) != -1)
                {
                    // HttpContext.Current.Response.Write(str_has+a_content+"<br/>__________________________");

                    regValue = new Regex("<[ ]*a[^<>]>", RegexOptions.IgnoreCase);
                    if (regValue.IsMatch(a_content))
                    {
                        a_content = a_content.Replace(regValue.Match(a_content).Value, "");
                    }
                    regValue = new Regex("<[ ]*a[ ]*/[ ]*>", RegexOptions.IgnoreCase);
                    if (regValue.IsMatch(a_content))
                    {
                        a_content = a_content.Replace(regValue.Match(a_content).Value, "");
                    }
                    a_content =Utils.RemoveHtml(a_content);
                    if (!lst.Contains(a_content))
                    {
                        if (maxListLength == 0)
                        {
                            lst.Add(a_content);
                        }
                        else
                        {
                            if (lst.Count <= maxListLength)
                            {
                                lst.Add(a_content);
                            }
                            else
                            {
                                break;
                            }
                        }
                    }
                }
            }

            return lst;
        }
        #region 获取指定字符串间的字符串
        /// <summary>
        /// 获取指定字符串间的字符串
        /// </summary>
        /// <param name="inputstring">获取源码</param>
        /// <param name="leftTag">开始标记</param>
        /// <param name="rightTag">结束标记</param>
        /// <param name="isHasHtml">是否包含html标记</param>
        /// <returns>返回字符串</returns>
        protected string GetCutString(string inputstring, string leftTag, string rightTag, bool isHasHtml)
        {
            reg = new Regex(leftTag + @"[\s|\S]*?" + rightTag, RegexOptions.IgnoreCase);
            string str = reg.Match(inputstring).Value;
            if (string.IsNullOrEmpty(str))
            {
                AppLog.Write("GetCutString-Match内容为空" + leftTag + rightTag + inputstring, AppLog.LogMessageType.Debug);
            }
            
                str = str.Replace(leftTag, "").Replace(rightTag, "");

            
            if (!isHasHtml)
            {
                str = Utils.RemoveHtml(str);
            }

            return str;

        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="inputstring"></param>
        /// <param name="cutStartNum"></param>
        /// <param name="cutEndNum"></param>
        /// <returns></returns>
        protected string GetCutString(string inputstring, int cutStartNum, int cutEndNum)
        {
            if (inputstring.Length > (cutStartNum + cutEndNum))
            {
                inputstring = inputstring.Substring(0, (inputstring.Length - cutEndNum));
                inputstring = inputstring.Substring(cutStartNum, (inputstring.Length - cutStartNum));
            }
            return inputstring;
        }
        #endregion
        /// <summary>
        /// 移除符合条件的字符串
        /// </summary>
        /// <param name="inputstring">要检测的字符串</param>
        /// <param name="RemoveRegex">移除的正则表达式</param>
        /// <returns>string</returns>
        public  string RemoveTag(string inputstring, string RemoveRegex)
        {
            reg = new Regex(RemoveRegex, RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            foreach (Match match in matchs)
            {
                inputstring = inputstring.Replace(match.Value, string.Empty);
            }
            return inputstring;
        }


        /// <summary>
        /// 移除符合条件的字符串
        /// </summary>
        /// <param name="inputstring">要检测的字符串</param>
        /// <param name="RemoveRegex">移除的正则表达式</param>
        /// <returns>string</returns>
        public string ReplaceTag(string inputstring, string ReplaceRegex, string ReplaceStr)
        {
            reg = new Regex(ReplaceRegex, RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputstring);
            foreach (Match match in matchs)
            {
                inputstring = inputstring.Replace(match.Value, ReplaceStr);
            }
            return inputstring;
        }
        #region 获取采集内容
        /// <summary>
        /// 获取采集内容
        /// </summary>
        /// <param name="url"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public Extend_CollectContentEntity GetContent(Extend_CollectSetInfoEntity setLinksInfo, string url)
        {
            Encoding coding = null;
            if (string.IsNullOrEmpty(setLinksInfo.ContentEncoding))
            {
                coding = Encoding.GetEncoding(setLinksInfo.Encoding);
            }
            else
                coding = Encoding.GetEncoding(setLinksInfo.ContentEncoding);
            string inputstring=string.Empty;
            try
            {
                inputstring = Utils.GetRequestString(url, 16000, 0, coding, (string.IsNullOrEmpty(setLinksInfo.Cookie) ? null : setLinksInfo.Cookie), setLinksInfo.FirstUrl);

            }
            catch (Exception ex)
            {
                AppLog.Write(ex.Message + url, AppLog.LogMessageType.Error);
            }
            return GetContent(setLinksInfo, inputstring, true);

        }

        /// <summary>
        /// 获取采集内容
        /// </summary>
        /// <param name="url"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public Extend_CollectContentEntity GetContent(Extend_CollectSetInfoEntity setLinksInfo, string content, bool isContent)
        {
            string inputstring = content;
           
            Extend_CollectContentEntity rCotent = new Extend_CollectContentEntity();
            if (string.IsNullOrEmpty(inputstring))
                return rCotent;
            if (setLinksInfo.TitleMatchLeft.Length < 1)
                rCotent.Title = GetFixTagContent(inputstring, "title");
            else
            {
                rCotent.Title = GetCutString(content, setLinksInfo.TitleMatchLeft, setLinksInfo.TitleMatchRight, false);
            }
            rCotent.Title = rCotent.Title.Trim();
          if (!string.IsNullOrEmpty(setLinksInfo.TitleReplaceStr))
              rCotent.Title = ReplaceArray(rCotent.Title, setLinksInfo.TitleReplaceStr);
          rCotent.Title = Utils.GetNotEmojiStr(rCotent.Title);
            //根据标题获取百度关键词
            if (setLinksInfo.IsAutoKeyWord)
            {
                //List<string> lstgoogle = new List<string>();
                //string PageUrl = "http://www.google.ee/search?q=" + rCotent.Title;
                //try
                //{
                //    string getHtml = Utils.GetRequestString(PageUrl, 800, 0, System.Text.Encoding.GetEncoding("utf-8"), (string.IsNullOrEmpty(setLinksInfo.Cookie) ? null : setLinksInfo.Cookie));

                //    lstgoogle = GetLinkText(getHtml, "s?wd", 3);
                //}
                //catch (Exception ex)
                //{
                //    AppLog.Write("谷歌采集关键词" + ex.Message, AppLog.LogMessageType.Error);
                //}
                string kewordTemp = Utils.GetRequestString("http://www.baidu.com/s?lm=0&si=&tn=yang0000meng_pg&rn=10&ie=utf-8&ct=0&wd=" + rCotent.Title + "&pn=10&ver=0&cl=3&uim=1&usm=0", 16000, 0, System.Text.Encoding.GetEncoding("utf-8"), (string.IsNullOrEmpty(setLinksInfo.Cookie) ? null : setLinksInfo.Cookie));
                List<string> keys = GetLinkText(kewordTemp, "s?wd", 1);
               // List<string> keys = Utils.CombineList(lstgoogle, lstbaidu);
                rCotent.Tags = Utils.ListToString(keys, ",", 4).Replace("[1]", string.Empty).Replace("[2]", string.Empty).Replace("[3]", string.Empty).Replace("[4]", string.Empty).Replace("[5]", string.Empty).Replace("[6]", string.Empty).Replace("[7]", string.Empty).Replace("[8]", string.Empty).Replace("[9]", string.Empty).Replace("[10]", string.Empty).Replace("上一页", string.Empty).Replace("[11]", string.Empty).Replace("下一页", string.Empty);

            }

            rCotent.TextContent = GetCutString(inputstring, setLinksInfo.ContentMatchLeft, setLinksInfo.ContentMatchRight, setLinksInfo.IsIncludeHtml);
            if (string.IsNullOrEmpty(rCotent.TextContent))
            {
                AppLog.Write(setLinksInfo.Name + "采集内容为空", AppLog.LogMessageType.Error);
                return rCotent;
            }
            
            //获取其他属性
            GetArticleOtherAttribute(inputstring, rCotent, setLinksInfo);
            if (setLinksInfo.IsIncludeHtml)
            {
                if (!(setLinksInfo.IsIncludeClass))
                {
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, "class[ ]*=['|\"]+[^'\"]+['|\"]+");
                    

                }
                //包含网址
                if (setLinksInfo.IsIncludeColor)
                {
                    Common.RegExpression.URLRegExpression URLRg = new CMN.Common.RegExpression.URLRegExpression();
                    Common.RegExpression.RegContext regContent = new CMN.Common.RegExpression.RegContext(rCotent.TextContent, 'R', "");
                    URLRg.Evaluate(regContent);
                    rCotent.TextContent = regContent.Context;
                   
                }
                
                    //if (!setLinksInfo.IsIncludeFile)
                    //{
                    //    rCotent.TextContent = RemoveTag(rCotent.TextContent, "color[ ]*:[^;'\"]+;");
                    //    rCotent.TextContent = RemoveTag(rCotent.TextContent, "color[ ]*:[^;'\"]+;");
                    //}
                    //if (!setLinksInfo.IsIncludeFrame)
                    //{
                    //    rCotent.TextContent = RemoveTag(rCotent.TextContent, "color[ ]*:[^;'\"]+;");
                    //    rCotent.TextContent = RemoveTag(rCotent.TextContent, "color[ ]*:[^;'\"]+;");
                    //}
                if (!(setLinksInfo.IsIncludeImg))
                {
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, "<[ ]*img[^>]*?>");

                }
                else
                {
                    rCotent.TextContent = ImgAddAlt(rCotent.TextContent, rCotent.Title);
                }

                if (!(setLinksInfo.IsIncludeLink))
                {
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, "<[ ]*a[ ]*[^>]*?>");
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, "<[ ]*/[ ]*a[ ]*[^<]*?>");


                }
                if (!(setLinksInfo.IsIncludeScript))
                {
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, @"<[ ]*script[^>]*?>[\s|\S]*?<[ ]*/[ ]*script[ ]*>");
                }
                if (!(setLinksInfo.IsIncludeStyle))
                {
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, @"<[ ]*style[^>]*?>[\s|\S]*?<[ ]*/[ ]*style[ ]*>");
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, "style[ ]*=['|\"]*[^\"|']*['|\"]");
                    rCotent.TextContent = RemoveTag(rCotent.TextContent, "color[ ]*=['|\"]+[^'\"]+['|\"]+");
                }

            }
           
           
           
            if (setLinksInfo.IsAutoImg)
            {
                rCotent.TextContent=  GetImages(rCotent.TextContent, setLinksInfo);
            }
           
            rCotent.TextContent = GetCutString(rCotent.TextContent, setLinksInfo.ContentRemoveLeftLength, setLinksInfo.ContentRemoveRightLength);
            if (setLinksInfo.ContentReplaseStr != "")
            {
                rCotent.TextContent = ReplaceArray(rCotent.TextContent, setLinksInfo.ContentReplaseStr);
                if (!string.IsNullOrEmpty(rCotent.Remark))
                {
                    rCotent.Remark = ReplaceArray2(rCotent.Remark, setLinksInfo.ContentReplaseStr);
                }
                if (!string.IsNullOrEmpty(rCotent.Editor))
                {
                    rCotent.Editor = ReplaceArray2(rCotent.Editor, setLinksInfo.ContentReplaseStr);
                }
            }
           
             rCotent.TextContent = ReplaceTag(rCotent.TextContent, "<[ ]*div[ ]*[^>]*?>", "<p>");
            rCotent.TextContent = ReplaceTag(rCotent.TextContent, "<[ ]*/[ ]*div[ ]*>", "</p>");
           // rCotent.TextContent = ReplaceTag(rCotent.TextContent, "<[ ]*p[ ]*[^>]*?>", "<p style=\"text-indent: 2em; text-align: left;\">");
             
            rCotent.TextContent = RemoveTag(rCotent.TextContent, @"<!--[\s|\S]*?-->");
            rCotent.TextContent = RemoveTag(rCotent.TextContent, "<[^>/]*>[ ]*<[ ]*/[^>]*>");
            rCotent.TextContent = Utils.HtmlDecode(rCotent.TextContent);
            rCotent.TextContent = Utils.GetNotEmojiStr(rCotent.TextContent);
            if (setLinksInfo.IsAutoDescription)
            {
                rCotent.Summary = GetSingleTagValue(inputstring, "meta", "name", "description");
                if (string.IsNullOrEmpty(rCotent.Summary))
                {
                    rCotent.Summary = Utils.GetLengthString(Utils.RemoveHtml(rCotent.TextContent), 200, false).Trim();
                }
                else
                {
                    rCotent.Summary = Utils.RemoveHtml(rCotent.Summary);

                }
                rCotent.Summary = ReplaceArray(rCotent.Summary, setLinksInfo.ContentReplaseStr);
            }
            else
            {
                rCotent.Summary = Utils.GetLengthString(Utils.RemoveHtml(rCotent.TextContent), 200, false).Trim();
            }
            rCotent.Summary = Utils.GetNotEmojiStr(rCotent.Summary);
            return rCotent;

        }
        private void GetArticleOtherAttribute(string inputstring,  Extend_CollectContentEntity rCotent, Extend_CollectSetInfoEntity setLinksInfo)
        {
            bool isHasRemark = false;
            if (!string.IsNullOrEmpty(setLinksInfo.FormatStr))
            {
                if (setLinksInfo.FormatStr.StartsWith("img"))
                {
                    regValue = new Regex(setLinksInfo.FormatStr.Remove(0, 3), RegexOptions.IgnoreCase);
                    Match match = regValue.Match(inputstring);
                    if (setLinksInfo.IsAutoImg)
                        rCotent.Img = LocalFileUrl(Utils.GetAbsolutePath(setLinksInfo.FirstUrl, match.Value));
                    else
                        rCotent.Img = Utils.GetAbsolutePath(setLinksInfo.FirstUrl, match.Value);

                }
                //获取第三页内容
                else if (setLinksInfo.FormatStr.StartsWith("remark"))
                {
                    isHasRemark = true;
                    string url2 = setLinksInfo.FormatStr;
                    string strParent = "";
                    string strFormat = "";
                    //SecondUrlRegex 为二部分，正则和组装内容 获取第二页面正则内容赋值给Remark
                    if (url2.IndexOf("&&&") > -1)
                    {
                        string[] strs = url2.Split(new string[] { "&&&" }, StringSplitOptions.RemoveEmptyEntries);
                        if (strs.Length == 2)
                        {
                            strParent = strs[0].Remove(0, 6);
                            strFormat = strs[1];
                            regValue = new Regex(strParent, RegexOptions.IgnoreCase);

                             url2 = regValue.Match(inputstring).Value;
                            url2 = Utils.GetAbsolutePath(setLinksInfo.FirstUrl, url2);
                            string strTemp2 = Utils.GetRequestString(url2, 16000, 1, Encoding.GetEncoding(setLinksInfo.ContentEncoding), null);
                            regValue = new Regex(strFormat, RegexOptions.IgnoreCase);
                            string content2 = regValue.Match(strTemp2).Value;
                            rCotent.Remark = content2;
                        }
                    }
                    

                      
                }
                else
                {
                    rCotent.TextContent = string.Format(setLinksInfo.FormatStr, rCotent.TextContent);
                }
            }


            #region 采集第二页，或属性
            if (!string.IsNullOrEmpty(setLinksInfo.SecondUrlRegex))
            {
                //根据第二个正则详细内容 给Remark
                if (setLinksInfo.SecondUrlRegex == "remark")
                {
                    regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                    Match match = regValue.Match(inputstring);

                    string content2 = match.Value;
                    rCotent.Remark = content2;
                    if (string.IsNullOrEmpty(rCotent.Remark))
                        rCotent.TextContent = "";
                }
                //根据第二个正则匹配多个详细内容，并组合 给Remark
                else if (setLinksInfo.SecondUrlRegex == "remarks")
                {
                    regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                    MatchCollection matchC = regValue.Matches(inputstring);
                    StringBuilder sb = new StringBuilder();
                    foreach (Match mc in matchC)
                    {
                        sb.AppendLine(mc.Value);
                        //, Environment.NewLine)
                    }
                    rCotent.Remark = sb.ToString();
                    if (string.IsNullOrEmpty(rCotent.Remark))
                        rCotent.TextContent = "";
                }
                //SecondContentRegex 为正则加组装内容 正则匹配后和第二部分合并赋值给Remark
                else if (setLinksInfo.SecondUrlRegex == "remarkAdd")
                {

                    string strParent = "";
                    string strFormat = "";
                    if (setLinksInfo.SecondContentRegex.IndexOf("&&&") > -1)
                    {
                        string[] strs = setLinksInfo.SecondContentRegex.Split(new string[] { "&&&" }, StringSplitOptions.RemoveEmptyEntries);
                        if (strs.Length == 2)
                        {
                            strParent = strs[0];
                            strFormat = strs[1];
                            regValue = new Regex(strParent, RegexOptions.IgnoreCase);
                            strFormat = string.Format(strFormat, regValue.Match(inputstring).Value);
                            rCotent.Remark = strFormat;
                        }
                    }

                    if (string.IsNullOrEmpty(rCotent.Remark))
                        rCotent.TextContent = "";
                }
                
                else if (setLinksInfo.SecondUrlRegex.StartsWith("remark"))
                {
                    string url2 = setLinksInfo.SecondUrlRegex;
                    string strParent = "";
                    string strFormat = "";
                    //SecondUrlRegex 为二部分，正则和组装内容 获取第二页面正则内容赋值给Remark
                    if (url2.IndexOf("&&&") > -1)
                    {
                        string[] strs = url2.Split(new string[] { "&&&" }, StringSplitOptions.RemoveEmptyEntries);
                        if (strs.Length >= 2)
                        {
                            strParent = strs[0].Remove(0, 6);
                            strFormat = strs[1];
                            regValue = new Regex(strParent, RegexOptions.IgnoreCase);

                            url2 = string.Format(strFormat, regValue.Match(inputstring).Value);
                            url2 = Utils.GetAbsolutePath(setLinksInfo.FirstUrl, url2);
                            string strTemp2 = Utils.GetRequestString(url2, 16000, 1, Encoding.GetEncoding(setLinksInfo.ContentEncoding), null);
                            regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                            string content2 = regValue.Match(strTemp2).Value;
                            if (strs.Length == 3)
                            {
                                if (content2.IndexOf(strs[2]) > -1)
                                    content2= "";
                            }
                            rCotent.Remark = content2;
                            if (string.IsNullOrEmpty(content2))
                                rCotent.TextContent = "";
                        }
                    }
                    //去除SecondUrlRegex remark 获取远程内容 根据SecondContentRegex 匹配内容赋值给Remark
                    else
                    {
                        regValue = new Regex(setLinksInfo.SecondUrlRegex.Remove(0, 6), RegexOptions.IgnoreCase);

                        url2 = regValue.Match(inputstring).Value;
                        url2 = Utils.GetAbsolutePath(setLinksInfo.FirstUrl, url2);
                        string strTemp2 = Utils.GetRequestString(url2, 16000, 1, Encoding.GetEncoding(setLinksInfo.ContentEncoding), null);
                        regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                        string content2 = regValue.Match(strTemp2).Value;
                        rCotent.Remark = content2;
                    }
                    if (string.IsNullOrEmpty(rCotent.Remark))
                        rCotent.TextContent = "";
                }
                //多个内容匹配
                else if (setLinksInfo.SecondUrlRegex.IndexOf("_|_") > -1)
                {
                    string[] strsUrlRegex = setLinksInfo.SecondUrlRegex.Split(new string[] { "_|_" }, StringSplitOptions.RemoveEmptyEntries);
                    string[] strsContentRegex = setLinksInfo.SecondContentRegex.Split(new string[] { "_|_" }, StringSplitOptions.RemoveEmptyEntries);
                    if (strsUrlRegex.Length == strsContentRegex.Length)
                    {
                        for (int i = 0; i < strsUrlRegex.Length; i++)
                        {

                            regValue = new Regex(strsContentRegex[i], RegexOptions.IgnoreCase);
                            Match match = regValue.Match(inputstring);

                            string content2 = match.Value;

                            if (strsUrlRegex[i] == "Size")
                                rCotent.Size = Utils.RemoveHtml(content2).Trim();

                            else if (strsUrlRegex[i] == "Author")
                                rCotent.Author = Utils.RemoveHtml(content2).Trim();
                            else if (strsUrlRegex[i] == "AddDate")
                                rCotent.AddDate = Utils.StringToDataTime( Utils.RemoveHtml(content2).Trim());

                            else if (strsUrlRegex[i] == "Editor")
                                rCotent.Editor =  Utils.RemoveHtml(content2).Trim();

                            else if (strsUrlRegex[i] == "Img")
                            {
                                if (!string.IsNullOrEmpty(content2))
                                {
                                    if (setLinksInfo.IsAutoImg)
                                        rCotent.Img = LocalFileUrl(Utils.GetAbsolutePath(setLinksInfo.FirstUrl, content2));
                                    else
                                        rCotent.Img = Utils.GetAbsolutePath(setLinksInfo.FirstUrl, content2);
                                }
                            }
                            else if (strsUrlRegex[i] == "Remark")
                            {
                                isHasRemark = true;
                                if (string.IsNullOrEmpty(content2))
                                    rCotent.TextContent = "";
                                rCotent.Remark = content2;

                            }

                        }
                    }
                    if (string.IsNullOrEmpty(rCotent.Remark.Trim()) && isHasRemark)
                        rCotent.TextContent = "";
                }
                    //获取编辑
                else if (setLinksInfo.SecondUrlRegex == "editor")
                {
                    regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                    Match match = regValue.Match(inputstring);

                    string content2 = match.Value;
                    rCotent.Editor = content2;
                    if (string.IsNullOrEmpty(rCotent.Editor))
                        rCotent.TextContent = "";

                }
                //多个内容匹配
                else if (setLinksInfo.SecondUrlRegex.IndexOf("&&&")>-1)
                {
                    string[] strsUrlRegex = setLinksInfo.SecondUrlRegex.Split(new string[] { "&&&" }, StringSplitOptions.RemoveEmptyEntries);
                    string[] strsContentRegex = setLinksInfo.SecondContentRegex.Split(new string[] { "&&&" }, StringSplitOptions.RemoveEmptyEntries);
                    if (strsUrlRegex.Length == strsContentRegex.Length)
                    {
                        for (int i = 0; i < strsUrlRegex.Length; i++)
                        {
                            regValue = new Regex(strsContentRegex[i], RegexOptions.IgnoreCase);
                            Match match = regValue.Match(inputstring);

                            string content2 = match.Value;

                            if (strsUrlRegex[i] == "Size")
                                rCotent.Size = content2;

                            else if (strsUrlRegex[i] == "Author")
                                rCotent.Author = content2;
                            else if (strsUrlRegex[i] == "AddDate")
                                rCotent.AddDate = Utils.StringToDataTime(content2);

                            else if (strsUrlRegex[i] == "Editor")
                                rCotent.Editor = content2;
                            else if (strsUrlRegex[i] == "Remark")
                                rCotent.Remark = content2;
                            
                        }
                    }
                }
                else
                {
                    //获取第二页地址 打开第二页 获取详细内容组合
                    regValue = new Regex(setLinksInfo.SecondUrlRegex, RegexOptions.IgnoreCase);
                    string url2 = regValue.Match(inputstring).Value;
                    url2 = Utils.GetAbsolutePath(setLinksInfo.FirstUrl, url2);
                    string strTemp2 = Utils.GetRequestString(url2, 16000, 1, Encoding.GetEncoding(setLinksInfo.ContentEncoding), null);
                    regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                    string content2 = regValue.Match(strTemp2).Value;
                    rCotent.TextContent = string.Concat(rCotent.TextContent, content2);
                }

            }
            else
            {
                //SecondUrlRegex 为空时，直接获取SecondContentRegex
                if (!string.IsNullOrEmpty(setLinksInfo.SecondContentRegex))
                {
                    regValue = new Regex(setLinksInfo.SecondContentRegex, RegexOptions.IgnoreCase);
                    string content2 = regValue.Match(inputstring).Value;
                    rCotent.TextContent = string.Concat(rCotent.TextContent, content2);

                }
            }
            #endregion
        }

        #endregion 

        /// <summary>
        /// 批量替换关键词 （abc=aaa;ccc=black)
        /// </summary>
        /// <param name="strOld"></param>
        /// <param name="strReplace"></param>
        /// <returns></returns>
        public static   string ReplaceArray(string strOld, string strReplace)
        {
            string[] strList = Utils.SplitString(strReplace, "\n");
            for (int i = 0; i < strList.Length; i++)
            {
                string[] strList2 = Utils.SplitString(strList[i], "=");
                if (strList2.Length == 2)
                {
                    if (strList2[1] != "blank")
                    {
                        strOld = Utils.ReplaceString(strOld, strList2[0], strList2[1], true);

                    }
                    else
                        strOld = Utils.ReplaceString(strOld, strList2[0], string.Empty, true);
                }
            }
            return strOld;
        }
        /// <summary>
        /// 批量替换关键词 （abc=aaa;ccc=black)
        /// </summary>
        /// <param name="strOld"></param>
        /// <param name="strReplace"></param>
        /// <returns></returns>
        protected string ReplaceArray2(string strOld, string strReplace)
        {
            string[] strList = Utils.SplitString(strReplace, "\n");
            for (int i = 0; i < strList.Length; i++)
            {
                string[] strList2 = Utils.SplitString(strList[i], "==");
                if (strList2.Length == 2)
                {
                    if (strList2[1] != "blank")
                    {
                        strOld = Utils.ReplaceString(strOld, strList2[0], strList2[1], true);

                    }
                    else
                        strOld = Utils.ReplaceString(strOld, strList2[0], string.Empty, true);
                }
            }
            return strOld;
        }
        #region 采集图片
        /// <summary>
        /// 图片新增alt属性
        /// </summary>
        /// <param name="inputString"></param>
        /// <param name="title"></param>
        /// <returns></returns>
        protected string ImgAddAlt(string inputString, string title)
        {
            reg = new Regex("<[ ]*img[^>]*?>", RegexOptions.IgnoreCase);
            MatchCollection matchs = reg.Matches(inputString);
            Regex altReg = new Regex("alt[ ]*=['|\"]*[^\"|']*['|\"]", RegexOptions.IgnoreCase);
               
            foreach (Match match in matchs)
            {
                if (!altReg.IsMatch(match.Value))
                {
                    if (match.Value.EndsWith("/>"))
                        inputString = inputString.Replace(match.Value, string.Concat(match.Value.Substring(0, match.Value.Length - 2), " alt=\"", title, "\" />"));
                    else
                        inputString = inputString.Replace(match.Value, string.Concat(match.Value.Substring(0, match.Value.Length - 1), " alt=\"", title, "\" />"));
                }
                else
                {
                    Match match2 = altReg.Match(match.Value);
                    if (match2.Value.Length < 10)
                        inputString = inputString.Replace(match.Value, match.Value.Replace(match2.Value, string.Concat(" alt=\"", title, "\" "))); ;
                }
            }
            return inputString;
        }

        protected string GetImages(string inputstring ,Extend_CollectSetInfoEntity setLinksInfo)
        {
            Common.RegExpression.ImageRegExpression imgRg = new Common.RegExpression.ImageRegExpression();
            Common.RegExpression.RegContext regContent = new CMN.Common.RegExpression.RegContext(inputstring, 'M', "");
            imgRg.Evaluate(regContent);
            Common.RegExpression.IRegExpression reg;
            Common.RegExpression.RegContext regTemp;
            string a_url = string.Empty;
            string TempStr = string.Empty;
            string TitleImg = string.Empty;
            foreach (string strmatch in regContent.Matches)
            {

                    if (TempStr != string.Empty)
                        TempStr += "$Array$" + strmatch.ToString();
                    else
                        TempStr = strmatch.ToString();

            }
            if (TempStr.Length == 0)
            {
                return inputstring;
            }
            string[] TempArr = TempStr.Split(new string[] { "$Array$" }, StringSplitOptions.None);
            TempStr = string.Empty;
           // Regex  imgReg = new Regex(@"src\s*=\s*.+?\.(gif|jpg|bmp|jpeg|psd|png)", RegexOptions.IgnoreCase);
            Regex imgReg = new Regex("src=['|\"]*[^\"|']*['|\"]", RegexOptions.IgnoreCase);
           
            for (int i = 0; i < TempArr.Length; i++)
            {
                MatchCollection  matches = imgReg.Matches(TempArr[i]);
                foreach (Match match in matches)
                {
                    if (TempStr != string.Empty)
                        TempStr += "$Array$" + match.ToString();
                    else
                        TempStr = match.ToString();
                }
            }
            if (TempStr.Length > 0)
            {
                imgReg = new Regex("src=['|\"]", RegexOptions.IgnoreCase);
                
                TempStr = imgReg.Replace(TempStr, "");
            }
            else
                return inputstring;
            
            TempStr = TempStr.Replace("\"", "");
            TempStr = TempStr.Replace("'", "");
            TempStr = TempStr.Replace(" ", "");

            //去掉重复图片
            TempArr = TempStr.Split(new string[] { "$Array$" }, StringSplitOptions.None);
            TempStr = string.Empty;
            for (int i = 0; i < TempArr.Length; i++)
            {
                if (TempStr.IndexOf(TempArr[i]) == -1)
                    TempStr += "$Array$" + TempArr[i];
            }
            if (TempStr.Length < 9)
            {
                return inputstring;
            }
            TempStr = TempStr.Substring(7);

            TempArr = TempStr.Split(new string[] { "$Array$" }, StringSplitOptions.None);
            TempStr = string.Empty;
            string ImageArr = string.Empty;
            for (int i = 0; i < TempArr.Length; i++)
            {
                imgReg = new Regex(TempArr[i]);
                Uri baseUri = new Uri(setLinksInfo.Url);
                string RemoteFileUrl = DefiniteUrl(TempArr[i], string.Concat(baseUri.Scheme, "://", baseUri.DnsSafeHost));
                string LocalPhotoUrl = LocalFileUrl( RemoteFileUrl);
                inputstring = imgReg.Replace(inputstring, LocalPhotoUrl);
               
            }

            return inputstring;

        }

       
        /// <summary>
        /// 远程图片本地化
        /// </summary>
        /// <param name="RemoteFileUrl"></param>
        /// <param name="webUrl">参考网站</param>
        /// <param name="SavePath">虚拟路径，以/结尾</param>
        /// <param name="isSave"></param>
        /// <returns></returns>
        public string LocalFileUrl(string RemoteFileUrl)
        {

            string SavePath = "/Upload/";
            string _LocalFileUrl = RemoteFileUrl;

            SavePath = SavePath + DateTime.Now.ToString("yyyy") + "/" + DateTime.Now.ToString("MM") + "/" + DateTime.Now.ToString("dd") + "/" + DateTime.Now.ToString("hh") + "/";
            Utils.CreateDir(Utils.GetMapPath(SavePath ));
            string fileType = _LocalFileUrl.Substring(_LocalFileUrl.LastIndexOf('.'));
            if (fileType.Split('?').Length > 0)
            {
                fileType = fileType.Split('?')[0];
            }
            System.Guid guid = new Guid();
            guid = Guid.NewGuid();
            SavePath += guid.ToString();

            SavePath += fileType;
            string saveAllPath=Utils.GetMapPath(SavePath  );
            if (SaveRemotePhoto(saveAllPath, _LocalFileUrl))
                _LocalFileUrl = SavePath;
            return _LocalFileUrl;
        }
        
        /// <summary>
        /// 取得实际地址
        /// </summary>
        /// <param name="PrimitiveUrl"></param>
        /// <param name="ConsultUrl"></param>
        /// <returns></returns>
        public string DefiniteUrl(string PrimitiveUrl, string ConsultUrl)
        {
            if (!ConsultUrl.StartsWith("http://"))
                ConsultUrl = "http://" + ConsultUrl;
            ConsultUrl = ConsultUrl.Replace("\\", "/");
            ConsultUrl = ConsultUrl.Replace("://", ":\\\\");
            PrimitiveUrl = PrimitiveUrl.Replace("\\", "/");

            if (ConsultUrl.Substring(ConsultUrl.Length - 1) != "/")
            {
                if (ConsultUrl.IndexOf('/') > 0)
                {
                    if (ConsultUrl.Substring(ConsultUrl.LastIndexOf("/"), ConsultUrl.Length - ConsultUrl.LastIndexOf("/")).IndexOf('.') == -1)//不含文件名，有缺陷，暂如此
                        ConsultUrl += "/";
                }
                else//直接是域名
                    ConsultUrl += "/";
            }
            string[] ConArray = ConsultUrl.Split('/');
            string returnStr = string.Empty;
            string[] PriArray;
            int pi = 0;
            if (PrimitiveUrl.Substring(0, 7) == "http://")
                returnStr = PrimitiveUrl.Replace("://", @":\\");
            else if (PrimitiveUrl.Substring(0, 1) == "/")//如果是绝对路径
                returnStr = ConArray[0] + PrimitiveUrl;
            else if (PrimitiveUrl.Substring(0, 2) == "./")//如果是当前路径
            {
                PrimitiveUrl = PrimitiveUrl.Substring(2, PrimitiveUrl.Length - 2);
                if (ConsultUrl.Substring(ConsultUrl.Length - 1) == "/")
                    returnStr = ConsultUrl + PrimitiveUrl;
                else
                    returnStr = ConsultUrl.Substring(0, ConsultUrl.LastIndexOf('/') + 1) + PrimitiveUrl;
            }
            else if (PrimitiveUrl.Substring(0, 3) == "../")//如果是相对父路径
            {
                while (PrimitiveUrl.Substring(0, 3) == "../")
                {
                    PrimitiveUrl = PrimitiveUrl.Substring(3);
                    pi++;
                }
                for (int i = 0; i < ConArray.Length - 1 - pi; i++)
                {
                    if (returnStr.Length > 0)
                        returnStr = returnStr + ConArray[i] + "/";
                    else
                        returnStr = ConArray[i] + "/";
                }
                returnStr = returnStr + PrimitiveUrl;
            }
            else//真实地址
            {
                if (PrimitiveUrl.IndexOf('/') > -1)
                {
                    PriArray = PrimitiveUrl.Split('/');
                    if (PriArray[0].IndexOf('.') > -1)
                    {
                        if (PrimitiveUrl.Substring(PrimitiveUrl.Length - 1) == "/")
                            returnStr = "http://" + PrimitiveUrl;
                        {
                            if (PriArray[PriArray.Length - 1].IndexOf('.') > -1)
                                returnStr = "http:\\" + PrimitiveUrl;
                            else
                                returnStr = "http:\\" + PrimitiveUrl + "/";
                        }
                    }
                    else
                    {
                        if (ConsultUrl.Substring(ConsultUrl.Length - 1) == "/")
                            returnStr = ConsultUrl + PrimitiveUrl;
                        else
                            returnStr = ConsultUrl.Substring(0, ConsultUrl.LastIndexOf('/') + 1) + PrimitiveUrl;
                    }
                }
                else
                {
                    if (PrimitiveUrl.IndexOf('.') > -1)
                    {
                        string lastUrl = ConsultUrl.Substring(ConsultUrl.LastIndexOf('.'));
                        if (ConsultUrl.Substring(ConsultUrl.Length - 1) == "/")
                        {
                            if (lastUrl == "com" || lastUrl == "cn" || lastUrl == "net" || lastUrl == "org")
                                returnStr = "http:\\" + PrimitiveUrl + "/";
                            else
                                returnStr = ConsultUrl + PrimitiveUrl;
                        }
                        else
                        {
                            if (lastUrl == "com" || lastUrl == "cn" || lastUrl == "net" || lastUrl == "org")
                                returnStr = "http:\\" + PrimitiveUrl + "/";
                            else
                                returnStr = ConsultUrl.Substring(0, ConsultUrl.LastIndexOf('/')) + "/" + PrimitiveUrl;
                        }
                    }
                    else
                    {
                        if (ConsultUrl.Substring(ConsultUrl.Length - 1) == "/")
                            returnStr = ConsultUrl + PrimitiveUrl + "/";
                        else
                            returnStr = ConsultUrl.Substring(0, ConsultUrl.LastIndexOf('/')) + "/" + PrimitiveUrl + "/";
                    }
                }
            }

            if (returnStr.Substring(0, 1) == "/")
                returnStr = returnStr.Substring(1);
            if (returnStr.Length > 0)
            {
                returnStr = returnStr.Replace("//", "/");
                returnStr = returnStr.Replace(@":\\", "://");
            }
            else
                returnStr = "$False$";
            return returnStr;
        }
        /// <summary>
        /// 抓取远程图片
        /// </summary>
        /// <param name="fileName">如果是要缩略图，记得图片名以_thumbs.jpg结尾</param>
        /// <param name="RemoteFileUrl"></param>
        /// <param name="width">最后的宽,0表示原来尺寸</param>
        /// <param name="height">最后的高,0表示原来尺寸</param>
        /// <returns></returns>
        public bool SaveRemotePhoto(string fileName, string RemoteFileUrl)
        {
            //try


            try
            {            //为防止图片未能本地化后，内网比较纠结，干脆抓取不到时直接报错
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(RemoteFileUrl);
                request.Timeout = 20000;
                request.KeepAlive = true;
                Uri baseUri = new Uri(RemoteFileUrl);
                request.Referer = baseUri.AbsoluteUri;
                HttpWebResponse HttpWResp = (HttpWebResponse)request.GetResponse();
                Stream stream = HttpWResp.GetResponseStream();
                //WebClient client = new System.Net.WebClient();
                // Stream stream = client.OpenRead(RemoteFileUrl);
                
                System.Drawing.Image getImage = System.Drawing.Image.FromStream(stream);
                if (getImage.Width > 50 && getImage.Height > 50)
                {
                    ImageHelper imgHelp = new ImageHelper();
                    int i=1;
                   
                    if (getImage.Width > 800 )
                    {
                         i= getImage.Width / 500;
                         if (i < 2)
                             i = 2;
                    }
                    
                    try
                    {
                        //try
                        //{

                        //    string path = string.Concat(Utils.GetServerPath(), "App_Data//font//elephant.ttf");
                        //    //读取字体文件
                        //    System.Drawing.Text.PrivateFontCollection pfc = new System.Drawing.Text.PrivateFontCollection();
                        //    pfc.AddFontFile(path);

                        //    //设置字体
                        //    System.Drawing.Font myFont = new System.Drawing.Font(pfc.Families[0], 6, (System.Drawing.FontStyle.Regular));

                        //    imgHelper.Makewater(getImage, CMN.Config.SiteConfigs.GetSiteConfig().Url, myFont, new System.Drawing.SolidBrush(System.Drawing.Color.White), new System.Drawing.Point(5, 60));
                        //}
                        //catch (Exception ex)
                        //{
                        //    AppLog.Write("生存水印错误" + ex.Message, AppLog.LogMessageType.Error, ex);
                        //}
                        CMN.Config.SiteConfigInfo siteInfo = CMN.Config.SiteConfigs.GetSiteConfig();
                        if (siteInfo.IsZhanQun)
                        {
                            imgHelp.ImgSaveLow(getImage, fileName, i);
                        }
                        else
                        {
                            Bitmap partImg = new Bitmap(getImage);
                            Graphics draw = Graphics.FromImage(partImg);
                            draw.DrawImage(getImage, 0, 0);
                            draw.Dispose();
                            partImg.Save(fileName, getImage.RawFormat);
                            partImg.Dispose();
                            
                          
                        }

                    }
                    catch (Exception ex2)
                    {
                        // AppLog.Write("采集图片错误ex2" + RemoteFileUrl + ":" + fileName, AppLog.LogMessageType.Error, ex2);
                        imgHelp.ImgSaveLow(getImage, fileName, i);
                    }
                    finally
                    {

                        getImage.Dispose();
                        stream.Dispose();
                        HttpWResp.Close();
                    }
                }
                getImage.Dispose();
                stream.Dispose();
                HttpWResp.Close();

            }
            catch (Exception ex)
            {
                AppLog.Write("采集图片错误" + RemoteFileUrl + fileName, AppLog.LogMessageType.Error, ex);
                return false;
            }
            finally
            {

            }
            return true;
        }
        #endregion
    }

  
}
